import requests
import bs4

def get_program_href(url):
    ori_content = requests.get(url).text
    soup = bs4.BeautifulSoup(ori_content)
    ul_tag = soup.find_all('ul')
    for subnavlist_tag in ul_tag:
        a_tag_all = subnavlist_tag.find_all('a')
        for a in a_tag_all:
            if 'Program' in a.get_text() and "ommittee" not in a.get_text():
                if a.get('href')[0] != '/':
                    return url + '/' + a.get('href')
                else:
                    return url + '/' + a.get('href').split('/')[-1]

    print(url + 'do not have Program tag')


def get_article_href(url):
    ori_content = requests.get(url).text
    soup = bs4.BeautifulSoup(ori_content)
    h2_tag = soup.find_all('h2', attrs = {"class": "node-title"})
    paper_url_list = []
    
    for item in h2_tag:
        label_a = item.find('a')
        if label_a is not None and '/presentation/' in label_a.get('href'):
            paper_url_list.append('https://www.usenix.org' + label_a.get('href'))

    return paper_url_list


def get_abstract_from_article_href(url):
    ori_content = requests.get(url).text
    soup = bs4.BeautifulSoup(ori_content)
    title = soup.find('h1').get_text()
    abstract = soup.find_all('div', attrs = {'class': 'field-item odd'})[1].get_text()
    return title + '\n' + abstract + '\n\n'

if __name__ == "__main__":
    for year in range(2008, 2018):
        program_url = get_program_href(f'http://conferences.sigcomm.org/sigcomm/{year}')
        print(program_url)
        ori_content = requests.get(program_url).text
        with open('sigcomm' + str(year) + '.html', 'w') as f:
            f.write(ori_content)

    # 18年之后的直接抓 http://conferences.sigcomm.org/sigcomm/2019/program.html